# -*-coding:utf-8 -*-
# BY jakejie@163.com
#2017.07.18
import requests
import random
from lxml import etree
import time
import itertools
import json
import codecs
import sys 
reload(sys) 
sys.setdefaultencoding( "utf-8" ) 
from fake_useragent import UserAgent
ua = UserAgent()
#生成请求链接，该处链接为搜索关键字为“公司”的url，修改页面数值即可
def create_url_list(page,steps):
	url_list = []
	for i in range(page,page+steps):
		url = 'https://s.1688.com/company/company_search.htm?keywords=%B9%AB%CB%BE&button_click=top&earseDirect=false&n=y&_source=sug&pageSize=30&offset=3&beginPage=' + str(i)
		url_list.append(url)
	return url_list
#根据请求url链接爬取页面，返回格式化好的页面代码
def requests_url_html(url):
	print "正在请求：%s"%url
	try:
		#请求头必须带cookie，不然获取不到页面信息，要求提示登录后再查看
		requests_headers = {
							#':authority':'s.1688.com',
							#':method':'GET',
							#':path':url,
							#':scheme':'https',
							'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
							'accept-encoding':'gzip, deflate, sdch',
							'accept-language':'zh-CN,zh;q=0.8',
							'cache-control':'max-age=0',
							#'cookie':'cna=ehW0EPRwY1sCAd3NLMm8rILI; l=As3NGpCdHkfiFprfeUahFMV4XeNCQgFY; UM_distinctid=15d48d3a5a326b-0927fd80f9464c-3e64430f-1fa400-15d48d3a5a43db; JSESSIONID=9L78Vhi82-cK3YHLYSrXNn9sAqt6-k4aMgPQ-spK; h_keys="%u516c%u53f8#%u7cbe%u5bc6%u673a%u68b0#%u7535%u8111"; cookie1=UtNZ%2Bpo0CtwwrOGPB1ZFtecUA9KqMQuT0Xedhdnj%2BpM%3D; cookie2=3f450900e6e1687c8c4131ba11b52a80; cookie17=UU2y9gO51%2FRhKQ%3D%3D; uss=VFXWsYgp7wepDlStZD3g2mdAahTOe6OxdVfpQtlG5en15fAROesJsdEdWA%3D%3D; t=b1867cae2ab33980397d5f91f0bc6379; _tb_token_=f384745b73d3d; sg=g56; __cn_logon__=true; __cn_logon_id__=15200228756wmg; ali_apache_track="c_ms=1|c_mid=b2b-2501087315|c_lid=15200228756wmg"; ali_apache_tracktmp="c_w_signed=Y"; cn_tmp="Z28mC+GqtZ2aR37b7UyfGShB22HrnEXkIcKiCfyEsHwn9b0ilD8MPoHpgao68AgqOZCi8Xtxa4ktPhbnZrhQ52JtL8nkyQo9Ez5hVqO6CqGOGfv7TGeEK6VInWiPKyEiBlvu45bgnKJs5vuy+khvSNLlqWUGWKTTbF/hUi7malm5zypNJ/ZV8fJMD/SM1cZBtVY2fqNsNih+hsY6GGO7VGk6F/O0Rpto1ldXL6h19eEHrio1t0nPIg=="; _cn_slid_=15wN0OQoNQ; tbsnid=%2FWk5qH9yhB%2BUGbAphiu%2B9rMjq4SQ0ePKsJ9oPjeEv%2Bk6sOlEpJKl9g%3D%3D; LoginUmid="oyoAdbrR%2BvQPMxoNIJsBMwtYMMWCt0IusI7pgJ5rDLh0ZvS19%2BaAOg%3D%3D"; userID="2sZnPyawLGik54yz7EtosA6tXymzNamGfMdxXcWCxoA6sOlEpJKl9g%3D%3D"; last_mid=b2b-2501087315; unb=2501087315; __last_loginid__=15200228756wmg; login="kFeyVBJLQQI%3D"; _csrf_token=1500335578665; ad_prefer="2017/07/18 07:52:59"; alisw=swIs1200%3D1%7C; userIDNum=jOLAS5iZvSbFSJDXntqVGw%3D%3D; _nk_=PtBtvHw%2BmpIX07Xtb%2BhWBQ%3D%3D; _is_show_loginId_change_block_=b2b-2501087315_false; ali_ab=58.20.21.247.1500165194394.0; _show_force_unbind_div_=b2b-2501087315_false; _show_sys_unbind_div_=b2b-2501087315_false; _show_user_unbind_div_=b2b-2501087315_false; __rn_alert__=false; _tmp_ck_0="4Zj%2Brf2I9e9HHzO5RKe2L5pfqY0AkSZPe9DkKjRQ7lojofSceItJgIJp1SUq2YtYWZkQWGJ34Pb0ShmpAXwWwySKqvBPfZBT15misSw%2BXt8Ly0NZo1h8emxMjNRwv5syyrMIfgMUImIyiVrI%2BeQBLHIFfUxpJH63m7tWUfJ0ENxLhWr591PMciJE412Ac7haY1s%2FSUryVg%2FBf9nNig2gTjzjz77l8pCQ7HC2T%2FMoGkTQapOYyTBgkbnzWcpXSqyxRNVReyVP4H%2FWyAPEi3Wxwrq3kT8%2F%2BPsdMZa3uUfmgBRprq1wqwrb3l0MsZHih9owlaOov2SJN9m1qmqRh%2BnQeKbHNEXJywLBxI4qNTNrJrA1mpGlklVye%2B2QC9EzkJWmVD1Gp3oyq%2B5u7c%2B93q0kDlm3UgAiM5PT8HMNSzX9CT7%2FzEBqvuMOcBp5%2BhvuPoQByDXRiHrINNcoOqNFUxbfcaB475dH%2Bq7osWhi3FJEoXzA43X5LGp6MeGtqOet1Ik6vQic9Q6ytLPyB8vfCHlFGA%3D%3D"; alicnweb=touch_tb_at%3D1500335628699%7Clastlogonid%3D15200228756wmg; isg=AgAA_8v8Ta54wzCRLVifi0LN0Y4SIfu2STI-E3qRzJuu9aAfIpm049ZHeULp',
							'cookie':'_uab_collina=150034325058835237353542; cna=ehW0EPRwY1sCAd3NLMm8rILI; l=As3NGpCdHkfiFprfeUahFMV4XeNCQgFY; UM_distinctid=15d48d3a5a326b-0927fd80f9464c-3e64430f-1fa400-15d48d3a5a43db; h_keys="%u516c%u53f8#%u7cbe%u5bc6%u673a%u68b0#%u7535%u8111"; ali_apache_track="c_ms=1|c_mid=b2b-2501087315|c_lid=15200228756wmg"; JSESSIONID=9L78dazw1-LM1Y4k8q1ydUk7YpJ6-8dAlgPQ-23o2; ad_prefer="2017/07/18 09:29:21"; ali_ab=58.20.21.247.1500165194394.0; _ITBU_IS_FIRST_VISITED_=n; cookie1=UtNZ%2Bpo0CtwwrOGPB1ZFtecUA9KqMQuT0Xedhdnj%2BpM%3D; cookie2=3d4eb74b1605ab8e719b005ae82257d4; cookie17=UU2y9gO51%2FRhKQ%3D%3D; uss=B0ens9SAMNec2GyuPU24IrWTxkV%2FVAyC0VNkUkhgUkmrC0VElsIakY8bgQ%3D%3D; t=b1867cae2ab33980397d5f91f0bc6379; _tb_token_=7b13e35eb355; sg=g56; __cn_logon__=true; __cn_logon_id__=15200228756wmg; ali_apache_tracktmp="c_w_signed=Y"; cn_tmp="Z28mC+GqtZ2aR37b7UyfGShB22HrnEXkIcKiCfyEsHwn9b0ilD8MPoHpgao68AgqOZCi8Xtxa4ktPhbnZrhQ541pg294SXUiI4VG6fCzqgHjVKLD2NBv9WRvmfBMTs023HkOnNwgt+H64WOdm41qPMUItojt35pYFLBN88XVUxlq5xx7qjk68ZLSaYlkeh9/UtjL4eQbPEL+qG46UqbX8D4yXqYkx3sl13sFai8IQ8/gbpHvv8I+Vw=="; _cn_slid_=15wN0OQoNQ; tbsnid=qBVgmO7lLbR9nPPci3r8DKUNyug334m81QlJDFAof5Q6sOlEpJKl9g%3D%3D; LoginUmid="oyoAdbrR%2BvQPMxoNIJsBMwtYMMWCt0IusI7pgJ5rDLh0ZvS19%2BaAOg%3D%3D"; userID="2sZnPyawLGik54yz7EtosA6tXymzNamGfMdxXcWCxoA6sOlEpJKl9g%3D%3D"; last_mid=b2b-2501087315; unb=2501087315; __last_loginid__=15200228756wmg; login="kFeyVBJLQQI%3D"; _csrf_token=1500343091092; _is_show_loginId_change_block_=b2b-2501087315_false; _show_force_unbind_div_=b2b-2501087315_false; _show_sys_unbind_div_=b2b-2501087315_false; _show_user_unbind_div_=b2b-2501087315_false; __rn_alert__=false; alicnweb=touch_tb_at%3D1500342904465%7Clastlogonid%3D15200228756wmg; isg=AtTUg6vhceqGtOSdGXzzz45hpRKGhefy9Q6KH26x_t_gWXWjlj0Ipr4bLU6d; userIDNum="jOLAS5iZvSbFSJDXntqVGw%3D%3D"; _tmp_ck_0="ljOqiJVw%2FMtr9kILFyZtIviQtEmwEyMCDeQFTF%2FbSle%2B89xI1AdpY1%2Fza2Z0Hi6zYrxOQeNjdMJuSbVV7LE9XlBn0r8idDox1BD%2FlS3MBxcMQ%2FUWQHlLAFJfjUCprsL45KOgnql56zsGfhZ%2B%2FuFYj3HG56Arm0DMeCf1FuzomJj4u7KlHa4spefWErFoEL%2BRi2MpHC1wc4B115MTlvAPKcarPj6QTpdoHMdsmcmXZHysiizrqXziRXLXTskd2MaVBHCrK30XqDwt%2FzXWVVaB%2B8R0iLh233R67V3Xmn14Ihjj90hz3qyfYVZG44nmhUn%2BWDQbK5A36YJsb5GEb3TqDrHxl8DrchLflb%2F%2Bp3u6GDb2%2BaYv%2FE%2FQVyWDS4TnfbUge%2Ba0qISaAcxRH%2BS6uNnTx7WS7ZJbaYgUIBHBpUiGu2xSA2G4aWllff2ucsRIxilKLcy%2BZDpnyoKqhhc8rWJfYv7tkL90YhC5YILRe1eaGlgGqXB1Y4Sb43N0ZqvAWAof%2BnHeUY7QiTycPjuDKDnxig%3D%3D"; _nk_="PtBtvHw%2BmpIX07Xtb%2BhWBQ%3D%3D"',
							'referer':'https://login.1688.com/member/marketSigninJump.htm?Done=https%3A%2F%2Fs.1688.com%2Fcompany%2Fcompany_search.htm%3Fkeywords%3D%25B9%25AB%25CB%25BE%26button_click%3Dtop%26earseDirect%3Dfalse%26n%3Dy%26_source%3Dsug%26pageSize%3D30%26offset%3D3%26beginPage%3D2',
							'upgrade-insecure-requests':'1',
							'user-agent':ua.random
							#'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
							}
		html = requests.get(url,headers = requests_headers)
		#print html.text
		tree = etree.HTML(html.text)
		time.sleep(random.uniform(3.5,5))
		image_url = tree.xpath('//*[@id="checkcodeImg"]/@src')
		if len(image_url) != 0:
			image_url = 'http:'+image_url[0]
			smTag = tree.xpath('//input[@name="smTag"]/@value')
			smsign = tree.xpath('//*[@id="query"]/input[9]/@value')
			print "出现验证码验证！！！"
			#保存验证码图片
			#html = requests.get(image_url)
			#with open('image.png','wb') as f:
			#	f.write(html.content)
			#f.close()
			return "False1"#,image_url,smTag[0],smsign[0]
		else:
			print "页面获取成功！"
			return tree
	except:
		print "页面获取失败！！！"
		return "False2"
#使用xpath方法，提取页面信息并保存json文件
def get_info_and_save(tree):
	#初始化文件保存位置
	file = codecs.open("E:\jiaocheng\F\Project\\taobao1688\info1\data\\company_info_test.json",'a',encoding = 'utf-8')
	#所有数据都在li节点下，先获取li节点，再遍历li节点
	infos = tree.xpath('//div[@id="sw_mod_searchlist"]/ul/li')
	print len(infos)
	#遍历所有的li节点，提取企业有效信息数据
	for info in infos:
		#time.sleep(0.1)
		#公司名称
		company = info.xpath('div[1]/div[2]/div[1]/a[1]/@title')

		#主营产品
		main_product = info.xpath('div[1]/div[2]/div[3]/div[1]/div[1]/a/span/text()')
		#对获取到的列表进行拼接，拼接成一个字符串
		product = "-*-".join(itertools.chain(main_product)).replace(';','')

		#所在地
		address = info.xpath('div[1]/div[2]/div[3]/div[1]/div[2]/a/text()')
		#判断提取到的信息是否为空，若为空，则赋值为“****”
		#由于部分企业某些项目缺少，该判断语句不可缺
		#下同
		if len(address)==0:
			address = ['****']

		#员工人数
		employee_num = info.xpath('div[1]/div[2]/div[3]/div[1]/div[3]/a/text()')
		if len(employee_num)==0:
			employee_num = ['****']

		#经营模式
		business_model = info.xpath('div[1]/div[2]/div[3]/div[2]/div[1]/b/text()')
		if len(business_model)==0:
			business_model = ['****']

		#工艺类型
		Process_types = info.xpath('div[1]/div[2]/div[3]/div[2]/div[2]/a/text()')
		if len(Process_types)==0:
			Process_types = ['****']
		#加工方式
		processing_method = info.xpath('div[1]/div[2]/div[3]/div[2]/div[3]/a/text()')
		if len(processing_method)==0:
			processing_method = ['****']

		#厂房面积
		factory_area = info.xpath('div[1]/div[2]/div[3]/div[2]/div[4]/a/text()')
		if len(factory_area)==0:
			factory_area = ['****']

		#print company,product,address,employee_num,business_model,Process_types,processing_method,factory_area
		print "*********************************"
		#保存获取到的数据
		#使用try...except...语句捕获写入异常
		try:
			dic = {
					"公司名称":company[0],
					"主营产品":product,
					"所在地":address[0],
					"员工人数":employee_num[0],
					"经营模式":business_model[0],
					"工艺类型":Process_types[0],
					"加工方式":processing_method[0],
					"厂房面积":factory_area[0],
					
				}
			dicts = json.dumps(dict(dic),ensure_ascii=False)
			line = dicts + '\n'
			file.write(line)
			print "数据写入成功！！！"
			#time.sleep(0.1)
		except:
			print "数据写入失败！！！"

#手动打码处理
def code_input(url):
	print 'http:'+url
	print "出现验证码验证，请输入上面链接图片的验证码！"
	code = raw_input('请输入验证码字符串：')
	return code
#使用浏览器刷新进行打码
def code_auto(url):
	#urls = url
	print "验证码，手动打码中...."
	print "请在浏览器端刷新网页，输入验证码！"
	print "请输入：\n\t\t1:浏览器可以正常访问\n\t\t2:浏览器无法正常访问\n"
	code = input("input:")
	if code == 1:
		TF = requests_url_html(url)
		if TF:
			print "******************请刷新浏览器重试****************"
			code_auto(url)
		else:
			print "访问正常，继续抓取中..."
			return True
	else:
		print "无法访问，请确保浏览器端可以正常访问！！！"
		code_auto()

#发送验证码打码数据，获取请求
def post_code(smtag,smsign,code,url):
	post_headers = {
					#':authority':'s.1688.com',
					#':method':'GET',
					#':path':url,
					#':scheme':'https',
					'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
					'accept-encoding':'gzip, deflate, sdch',
					'accept-language':'zh-CN,zh;q=0.8',
					'cache-control':'max-age=0',
					#'cookie':'cna=ehW0EPRwY1sCAd3NLMm8rILI; l=As3NGpCdHkfiFprfeUahFMV4XeNCQgFY; UM_distinctid=15d48d3a5a326b-0927fd80f9464c-3e64430f-1fa400-15d48d3a5a43db; JSESSIONID=9L78Vhi82-cK3YHLYSrXNn9sAqt6-k4aMgPQ-spK; h_keys="%u516c%u53f8#%u7cbe%u5bc6%u673a%u68b0#%u7535%u8111"; cookie1=UtNZ%2Bpo0CtwwrOGPB1ZFtecUA9KqMQuT0Xedhdnj%2BpM%3D; cookie2=3f450900e6e1687c8c4131ba11b52a80; cookie17=UU2y9gO51%2FRhKQ%3D%3D; uss=VFXWsYgp7wepDlStZD3g2mdAahTOe6OxdVfpQtlG5en15fAROesJsdEdWA%3D%3D; t=b1867cae2ab33980397d5f91f0bc6379; _tb_token_=f384745b73d3d; sg=g56; __cn_logon__=true; __cn_logon_id__=15200228756wmg; ali_apache_track="c_ms=1|c_mid=b2b-2501087315|c_lid=15200228756wmg"; ali_apache_tracktmp="c_w_signed=Y"; cn_tmp="Z28mC+GqtZ2aR37b7UyfGShB22HrnEXkIcKiCfyEsHwn9b0ilD8MPoHpgao68AgqOZCi8Xtxa4ktPhbnZrhQ52JtL8nkyQo9Ez5hVqO6CqGOGfv7TGeEK6VInWiPKyEiBlvu45bgnKJs5vuy+khvSNLlqWUGWKTTbF/hUi7malm5zypNJ/ZV8fJMD/SM1cZBtVY2fqNsNih+hsY6GGO7VGk6F/O0Rpto1ldXL6h19eEHrio1t0nPIg=="; _cn_slid_=15wN0OQoNQ; tbsnid=%2FWk5qH9yhB%2BUGbAphiu%2B9rMjq4SQ0ePKsJ9oPjeEv%2Bk6sOlEpJKl9g%3D%3D; LoginUmid="oyoAdbrR%2BvQPMxoNIJsBMwtYMMWCt0IusI7pgJ5rDLh0ZvS19%2BaAOg%3D%3D"; userID="2sZnPyawLGik54yz7EtosA6tXymzNamGfMdxXcWCxoA6sOlEpJKl9g%3D%3D"; last_mid=b2b-2501087315; unb=2501087315; __last_loginid__=15200228756wmg; login="kFeyVBJLQQI%3D"; _csrf_token=1500335578665; ad_prefer="2017/07/18 07:52:59"; alisw=swIs1200%3D1%7C; userIDNum=jOLAS5iZvSbFSJDXntqVGw%3D%3D; _nk_=PtBtvHw%2BmpIX07Xtb%2BhWBQ%3D%3D; _is_show_loginId_change_block_=b2b-2501087315_false; ali_ab=58.20.21.247.1500165194394.0; _show_force_unbind_div_=b2b-2501087315_false; _show_sys_unbind_div_=b2b-2501087315_false; _show_user_unbind_div_=b2b-2501087315_false; __rn_alert__=false; _tmp_ck_0="4Zj%2Brf2I9e9HHzO5RKe2L5pfqY0AkSZPe9DkKjRQ7lojofSceItJgIJp1SUq2YtYWZkQWGJ34Pb0ShmpAXwWwySKqvBPfZBT15misSw%2BXt8Ly0NZo1h8emxMjNRwv5syyrMIfgMUImIyiVrI%2BeQBLHIFfUxpJH63m7tWUfJ0ENxLhWr591PMciJE412Ac7haY1s%2FSUryVg%2FBf9nNig2gTjzjz77l8pCQ7HC2T%2FMoGkTQapOYyTBgkbnzWcpXSqyxRNVReyVP4H%2FWyAPEi3Wxwrq3kT8%2F%2BPsdMZa3uUfmgBRprq1wqwrb3l0MsZHih9owlaOov2SJN9m1qmqRh%2BnQeKbHNEXJywLBxI4qNTNrJrA1mpGlklVye%2B2QC9EzkJWmVD1Gp3oyq%2B5u7c%2B93q0kDlm3UgAiM5PT8HMNSzX9CT7%2FzEBqvuMOcBp5%2BhvuPoQByDXRiHrINNcoOqNFUxbfcaB475dH%2Bq7osWhi3FJEoXzA43X5LGp6MeGtqOet1Ik6vQic9Q6ytLPyB8vfCHlFGA%3D%3D"; alicnweb=touch_tb_at%3D1500335628699%7Clastlogonid%3D15200228756wmg; isg=AgAA_8v8Ta54wzCRLVifi0LN0Y4SIfu2STI-E3qRzJuu9aAfIpm049ZHeULp',
					'cookie':'_uab_collina=150034325058835237353542; cna=ehW0EPRwY1sCAd3NLMm8rILI; l=As3NGpCdHkfiFprfeUahFMV4XeNCQgFY; UM_distinctid=15d48d3a5a326b-0927fd80f9464c-3e64430f-1fa400-15d48d3a5a43db; h_keys="%u516c%u53f8#%u7cbe%u5bc6%u673a%u68b0#%u7535%u8111"; ali_apache_track="c_ms=1|c_mid=b2b-2501087315|c_lid=15200228756wmg"; JSESSIONID=9L78dazw1-LM1Y4k8q1ydUk7YpJ6-8dAlgPQ-23o2; ad_prefer="2017/07/18 09:29:21"; ali_ab=58.20.21.247.1500165194394.0; _ITBU_IS_FIRST_VISITED_=n; cookie1=UtNZ%2Bpo0CtwwrOGPB1ZFtecUA9KqMQuT0Xedhdnj%2BpM%3D; cookie2=3d4eb74b1605ab8e719b005ae82257d4; cookie17=UU2y9gO51%2FRhKQ%3D%3D; uss=B0ens9SAMNec2GyuPU24IrWTxkV%2FVAyC0VNkUkhgUkmrC0VElsIakY8bgQ%3D%3D; t=b1867cae2ab33980397d5f91f0bc6379; _tb_token_=7b13e35eb355; sg=g56; __cn_logon__=true; __cn_logon_id__=15200228756wmg; ali_apache_tracktmp="c_w_signed=Y"; cn_tmp="Z28mC+GqtZ2aR37b7UyfGShB22HrnEXkIcKiCfyEsHwn9b0ilD8MPoHpgao68AgqOZCi8Xtxa4ktPhbnZrhQ541pg294SXUiI4VG6fCzqgHjVKLD2NBv9WRvmfBMTs023HkOnNwgt+H64WOdm41qPMUItojt35pYFLBN88XVUxlq5xx7qjk68ZLSaYlkeh9/UtjL4eQbPEL+qG46UqbX8D4yXqYkx3sl13sFai8IQ8/gbpHvv8I+Vw=="; _cn_slid_=15wN0OQoNQ; tbsnid=qBVgmO7lLbR9nPPci3r8DKUNyug334m81QlJDFAof5Q6sOlEpJKl9g%3D%3D; LoginUmid="oyoAdbrR%2BvQPMxoNIJsBMwtYMMWCt0IusI7pgJ5rDLh0ZvS19%2BaAOg%3D%3D"; userID="2sZnPyawLGik54yz7EtosA6tXymzNamGfMdxXcWCxoA6sOlEpJKl9g%3D%3D"; last_mid=b2b-2501087315; unb=2501087315; __last_loginid__=15200228756wmg; login="kFeyVBJLQQI%3D"; _csrf_token=1500343091092; _is_show_loginId_change_block_=b2b-2501087315_false; _show_force_unbind_div_=b2b-2501087315_false; _show_sys_unbind_div_=b2b-2501087315_false; _show_user_unbind_div_=b2b-2501087315_false; __rn_alert__=false; alicnweb=touch_tb_at%3D1500342904465%7Clastlogonid%3D15200228756wmg; isg=AtTUg6vhceqGtOSdGXzzz45hpRKGhefy9Q6KH26x_t_gWXWjlj0Ipr4bLU6d; userIDNum="jOLAS5iZvSbFSJDXntqVGw%3D%3D"; _tmp_ck_0="ljOqiJVw%2FMtr9kILFyZtIviQtEmwEyMCDeQFTF%2FbSle%2B89xI1AdpY1%2Fza2Z0Hi6zYrxOQeNjdMJuSbVV7LE9XlBn0r8idDox1BD%2FlS3MBxcMQ%2FUWQHlLAFJfjUCprsL45KOgnql56zsGfhZ%2B%2FuFYj3HG56Arm0DMeCf1FuzomJj4u7KlHa4spefWErFoEL%2BRi2MpHC1wc4B115MTlvAPKcarPj6QTpdoHMdsmcmXZHysiizrqXziRXLXTskd2MaVBHCrK30XqDwt%2FzXWVVaB%2B8R0iLh233R67V3Xmn14Ihjj90hz3qyfYVZG44nmhUn%2BWDQbK5A36YJsb5GEb3TqDrHxl8DrchLflb%2F%2Bp3u6GDb2%2BaYv%2FE%2FQVyWDS4TnfbUge%2Ba0qISaAcxRH%2BS6uNnTx7WS7ZJbaYgUIBHBpUiGu2xSA2G4aWllff2ucsRIxilKLcy%2BZDpnyoKqhhc8rWJfYv7tkL90YhC5YILRe1eaGlgGqXB1Y4Sb43N0ZqvAWAof%2BnHeUY7QiTycPjuDKDnxig%3D%3D"; _nk_="PtBtvHw%2BmpIX07Xtb%2BhWBQ%3D%3D"',
					'referer':'https://login.1688.com/member/marketSigninJump.htm?Done=https%3A%2F%2Fs.1688.com%2Fcompany%2Fcompany_search.htm%3Fkeywords%3D%25B9%25AB%25CB%25BE%26button_click%3Dtop%26earseDirect%3Dfalse%26n%3Dy%26_source%3Dsug%26pageSize%3D30%26offset%3D3%26beginPage%3D2',
					'upgrade-insecure-requests':'1',
					'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
					}
	requests_headers = {
							#':authority':'s.1688.com',
							#':method':'GET',
							#':path':url,
							#':scheme':'https',
							'accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
							'accept-encoding':'gzip, deflate, sdch',
							'accept-language':'zh-CN,zh;q=0.8',
							'cache-control':'max-age=0',
							#'cookie':'cna=ehW0EPRwY1sCAd3NLMm8rILI; l=As3NGpCdHkfiFprfeUahFMV4XeNCQgFY; UM_distinctid=15d48d3a5a326b-0927fd80f9464c-3e64430f-1fa400-15d48d3a5a43db; JSESSIONID=9L78Vhi82-cK3YHLYSrXNn9sAqt6-k4aMgPQ-spK; h_keys="%u516c%u53f8#%u7cbe%u5bc6%u673a%u68b0#%u7535%u8111"; cookie1=UtNZ%2Bpo0CtwwrOGPB1ZFtecUA9KqMQuT0Xedhdnj%2BpM%3D; cookie2=3f450900e6e1687c8c4131ba11b52a80; cookie17=UU2y9gO51%2FRhKQ%3D%3D; uss=VFXWsYgp7wepDlStZD3g2mdAahTOe6OxdVfpQtlG5en15fAROesJsdEdWA%3D%3D; t=b1867cae2ab33980397d5f91f0bc6379; _tb_token_=f384745b73d3d; sg=g56; __cn_logon__=true; __cn_logon_id__=15200228756wmg; ali_apache_track="c_ms=1|c_mid=b2b-2501087315|c_lid=15200228756wmg"; ali_apache_tracktmp="c_w_signed=Y"; cn_tmp="Z28mC+GqtZ2aR37b7UyfGShB22HrnEXkIcKiCfyEsHwn9b0ilD8MPoHpgao68AgqOZCi8Xtxa4ktPhbnZrhQ52JtL8nkyQo9Ez5hVqO6CqGOGfv7TGeEK6VInWiPKyEiBlvu45bgnKJs5vuy+khvSNLlqWUGWKTTbF/hUi7malm5zypNJ/ZV8fJMD/SM1cZBtVY2fqNsNih+hsY6GGO7VGk6F/O0Rpto1ldXL6h19eEHrio1t0nPIg=="; _cn_slid_=15wN0OQoNQ; tbsnid=%2FWk5qH9yhB%2BUGbAphiu%2B9rMjq4SQ0ePKsJ9oPjeEv%2Bk6sOlEpJKl9g%3D%3D; LoginUmid="oyoAdbrR%2BvQPMxoNIJsBMwtYMMWCt0IusI7pgJ5rDLh0ZvS19%2BaAOg%3D%3D"; userID="2sZnPyawLGik54yz7EtosA6tXymzNamGfMdxXcWCxoA6sOlEpJKl9g%3D%3D"; last_mid=b2b-2501087315; unb=2501087315; __last_loginid__=15200228756wmg; login="kFeyVBJLQQI%3D"; _csrf_token=1500335578665; ad_prefer="2017/07/18 07:52:59"; alisw=swIs1200%3D1%7C; userIDNum=jOLAS5iZvSbFSJDXntqVGw%3D%3D; _nk_=PtBtvHw%2BmpIX07Xtb%2BhWBQ%3D%3D; _is_show_loginId_change_block_=b2b-2501087315_false; ali_ab=58.20.21.247.1500165194394.0; _show_force_unbind_div_=b2b-2501087315_false; _show_sys_unbind_div_=b2b-2501087315_false; _show_user_unbind_div_=b2b-2501087315_false; __rn_alert__=false; _tmp_ck_0="4Zj%2Brf2I9e9HHzO5RKe2L5pfqY0AkSZPe9DkKjRQ7lojofSceItJgIJp1SUq2YtYWZkQWGJ34Pb0ShmpAXwWwySKqvBPfZBT15misSw%2BXt8Ly0NZo1h8emxMjNRwv5syyrMIfgMUImIyiVrI%2BeQBLHIFfUxpJH63m7tWUfJ0ENxLhWr591PMciJE412Ac7haY1s%2FSUryVg%2FBf9nNig2gTjzjz77l8pCQ7HC2T%2FMoGkTQapOYyTBgkbnzWcpXSqyxRNVReyVP4H%2FWyAPEi3Wxwrq3kT8%2F%2BPsdMZa3uUfmgBRprq1wqwrb3l0MsZHih9owlaOov2SJN9m1qmqRh%2BnQeKbHNEXJywLBxI4qNTNrJrA1mpGlklVye%2B2QC9EzkJWmVD1Gp3oyq%2B5u7c%2B93q0kDlm3UgAiM5PT8HMNSzX9CT7%2FzEBqvuMOcBp5%2BhvuPoQByDXRiHrINNcoOqNFUxbfcaB475dH%2Bq7osWhi3FJEoXzA43X5LGp6MeGtqOet1Ik6vQic9Q6ytLPyB8vfCHlFGA%3D%3D"; alicnweb=touch_tb_at%3D1500335628699%7Clastlogonid%3D15200228756wmg; isg=AgAA_8v8Ta54wzCRLVifi0LN0Y4SIfu2STI-E3qRzJuu9aAfIpm049ZHeULp',
							'cookie':'_uab_collina=150034325058835237353542; cna=ehW0EPRwY1sCAd3NLMm8rILI; l=As3NGpCdHkfiFprfeUahFMV4XeNCQgFY; UM_distinctid=15d48d3a5a326b-0927fd80f9464c-3e64430f-1fa400-15d48d3a5a43db; h_keys="%u516c%u53f8#%u7cbe%u5bc6%u673a%u68b0#%u7535%u8111"; ali_apache_track="c_ms=1|c_mid=b2b-2501087315|c_lid=15200228756wmg"; JSESSIONID=9L78dazw1-LM1Y4k8q1ydUk7YpJ6-8dAlgPQ-23o2; ad_prefer="2017/07/18 09:29:21"; ali_ab=58.20.21.247.1500165194394.0; _ITBU_IS_FIRST_VISITED_=n; cookie1=UtNZ%2Bpo0CtwwrOGPB1ZFtecUA9KqMQuT0Xedhdnj%2BpM%3D; cookie2=3d4eb74b1605ab8e719b005ae82257d4; cookie17=UU2y9gO51%2FRhKQ%3D%3D; uss=B0ens9SAMNec2GyuPU24IrWTxkV%2FVAyC0VNkUkhgUkmrC0VElsIakY8bgQ%3D%3D; t=b1867cae2ab33980397d5f91f0bc6379; _tb_token_=7b13e35eb355; sg=g56; __cn_logon__=true; __cn_logon_id__=15200228756wmg; ali_apache_tracktmp="c_w_signed=Y"; cn_tmp="Z28mC+GqtZ2aR37b7UyfGShB22HrnEXkIcKiCfyEsHwn9b0ilD8MPoHpgao68AgqOZCi8Xtxa4ktPhbnZrhQ541pg294SXUiI4VG6fCzqgHjVKLD2NBv9WRvmfBMTs023HkOnNwgt+H64WOdm41qPMUItojt35pYFLBN88XVUxlq5xx7qjk68ZLSaYlkeh9/UtjL4eQbPEL+qG46UqbX8D4yXqYkx3sl13sFai8IQ8/gbpHvv8I+Vw=="; _cn_slid_=15wN0OQoNQ; tbsnid=qBVgmO7lLbR9nPPci3r8DKUNyug334m81QlJDFAof5Q6sOlEpJKl9g%3D%3D; LoginUmid="oyoAdbrR%2BvQPMxoNIJsBMwtYMMWCt0IusI7pgJ5rDLh0ZvS19%2BaAOg%3D%3D"; userID="2sZnPyawLGik54yz7EtosA6tXymzNamGfMdxXcWCxoA6sOlEpJKl9g%3D%3D"; last_mid=b2b-2501087315; unb=2501087315; __last_loginid__=15200228756wmg; login="kFeyVBJLQQI%3D"; _csrf_token=1500343091092; _is_show_loginId_change_block_=b2b-2501087315_false; _show_force_unbind_div_=b2b-2501087315_false; _show_sys_unbind_div_=b2b-2501087315_false; _show_user_unbind_div_=b2b-2501087315_false; __rn_alert__=false; alicnweb=touch_tb_at%3D1500342904465%7Clastlogonid%3D15200228756wmg; isg=AtTUg6vhceqGtOSdGXzzz45hpRKGhefy9Q6KH26x_t_gWXWjlj0Ipr4bLU6d; userIDNum="jOLAS5iZvSbFSJDXntqVGw%3D%3D"; _tmp_ck_0="ljOqiJVw%2FMtr9kILFyZtIviQtEmwEyMCDeQFTF%2FbSle%2B89xI1AdpY1%2Fza2Z0Hi6zYrxOQeNjdMJuSbVV7LE9XlBn0r8idDox1BD%2FlS3MBxcMQ%2FUWQHlLAFJfjUCprsL45KOgnql56zsGfhZ%2B%2FuFYj3HG56Arm0DMeCf1FuzomJj4u7KlHa4spefWErFoEL%2BRi2MpHC1wc4B115MTlvAPKcarPj6QTpdoHMdsmcmXZHysiizrqXziRXLXTskd2MaVBHCrK30XqDwt%2FzXWVVaB%2B8R0iLh233R67V3Xmn14Ihjj90hz3qyfYVZG44nmhUn%2BWDQbK5A36YJsb5GEb3TqDrHxl8DrchLflb%2F%2Bp3u6GDb2%2BaYv%2FE%2FQVyWDS4TnfbUge%2Ba0qISaAcxRH%2BS6uNnTx7WS7ZJbaYgUIBHBpUiGu2xSA2G4aWllff2ucsRIxilKLcy%2BZDpnyoKqhhc8rWJfYv7tkL90YhC5YILRe1eaGlgGqXB1Y4Sb43N0ZqvAWAof%2BnHeUY7QiTycPjuDKDnxig%3D%3D"; _nk_="PtBtvHw%2BmpIX07Xtb%2BhWBQ%3D%3D"',
							'referer':'https://login.1688.com/member/marketSigninJump.htm?Done=https%3A%2F%2Fs.1688.com%2Fcompany%2Fcompany_search.htm%3Fkeywords%3D%25B9%25AB%25CB%25BE%26button_click%3Dtop%26earseDirect%3Dfalse%26n%3Dy%26_source%3Dsug%26pageSize%3D30%26offset%3D3%26beginPage%3D2',
							'upgrade-insecure-requests':'1',
							'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
							}
	from_data = {
			"ua":"131UW5TcyMNYQwiAiwZTXFIdUh1SHJOe0BuOG4=|Um5Ockp3SnRBeUB0THRMeS8=|U2xMHDJ7G2AHYg8hAS8XKgQkClY3UT1aJF5wJnA=|VGhXd1llXWBdY1ZuV2NbY1pmUWxOekZyRn1GckxySnNMd0h9Qmw6|VWldfS0SMg0wCCgRMR8pXTxSfCp8|VmhIGCUFJBgkBDgFPwIiGyUdPQE1CjcXKxYjHj4CNgk0FCgVKxZAFg==|V25OHjAePgI3AyMYLAw2DDBmMA==|WGFBET8RMQ43AyMfIB09CDcNWw0=|WWBAED4QMA86BycbIhY2AjwHUQc=|WmFBET8RMQ0yDCwRMQ4xDjdhNw==|W2BAED4QMAwzDS0QMA86DzFnMQ==|XGREFDpRMVw3VD9YNFkafAV5BCoKWmFUakp1SnchATwcMhw8BzkNMWcx|XWZGFjhTM141Vj1aNlsYfgd7BigIMQQkGTkDOwM8ajw=|XmZGFjgWNmZdZ1x8Q3ZLHT0AIA4gADQNMwdRBw==|X2ZbZkZ7W2REeEF9XWNbYUF4WGRZeU1tWHhHfV1iWXlFfFxjWnpGeS8=",
			"action":"QueryAction",
			"event_submit_do_query":"ok",
			"smPolicy":"searchweb2-company-anti_Spider-html-checkcode",
			"smReturn":url,
			"smApp":"searchweb2",
			"smCharset":"GBK",
			"smTag":smtag,
			"smSign":smsign,
			"identity":"sm-searchweb2",
			#"captcha":,
			"checkcode":code,
				}
	html = requests.post(url,headers = post_headers,data = from_data)
	#print html.text
	print "***************************"
	html = requests.get(url,requests_headers)
	#print html.text
	tree = etree.HTML(html.text)
	return tree
#主函数
def main():
	#生成需要请求的页面列表
	steps = 100
	for page in range(1,10000000,steps):
		urls = create_url_list(page,steps)
		for url in urls:
			tree = requests_url_html(url)
			if tree == "False1":
				code = code_auto(url)
				if code == True:
					#tree = requests_url_html(url)
					get_info_and_save(tree)
				#image_url = tree[1]
				#smtag = tree[2]
				#smsign = tree[3]
				#code = code_input(image_url)
				#print image_url,smtag,smsign,code
				#tree = post_code(smtag,smsign,code,url)
				#get_info_and_save(tree)
			elif tree == "False2":
				print "页面获取失败，链接为：%s"%url
				#保存没有获取到页面的url链接
				with open('Error_url_list.txt','a') as f:
					line = url + '\n'
					f.write(line)
				f.close()
			else:
				get_info_and_save(tree)

if __name__ == '__main__':
	main()